{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b3d90105-c40d-4376-8e9b-8f4ba51b313a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "页面加载完成，开始抓取...\n",
      "正在处理第 1 页...\n",
      "正在处理第 2 页...\n",
      "正在处理第 3 页...\n",
      "翻页失败: Message: \n",
      "\n",
      "数据已保存至 Hedysarum_Components_Corrected茯苓.csv\n"
     ]
    }
   ],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.chrome.service import Service\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from webdriver_manager.chrome import ChromeDriverManager\n",
    "from selenium.common.exceptions import NoSuchElementException, TimeoutException\n",
    "import pandas as pd\n",
    "import time\n",
    "\n",
    "def scrape_tcmsp(url, max_pages=64):\n",
    "    driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()))\n",
    "    all_data = []\n",
    "    \n",
    "    try:\n",
    "        driver.get(url)\n",
    "        print(\"页面加载完成，开始抓取...\")\n",
    "        \n",
    "        WebDriverWait(driver, 30).until(\n",
    "            EC.presence_of_element_located((By.CSS_SELECTOR, \"div.k-grid-content table\"))\n",
    "        )\n",
    "        \n",
    "        current_page = 1\n",
    "        while current_page <= max_pages:\n",
    "            print(f\"正在处理第 {current_page} 页...\")\n",
    "            \n",
    "            rows = WebDriverWait(driver, 30).until(\n",
    "                EC.presence_of_all_elements_located((By.CSS_SELECTOR, \"tr[data-uid][role='row']\"))\n",
    "            )\n",
    "            \n",
    "            for row in rows:\n",
    "                try:\n",
    "                    cells = row.find_elements(By.CSS_SELECTOR, \"td[role='gridcell']\")\n",
    "                    if len(cells) < 12:\n",
    "                        continue\n",
    "                    \n",
    "                    # ========== 修复点1: Molecule Name提取 ==========\n",
    "                    mol_id = cells[0].text.strip()\n",
    "                    \n",
    "                    # 检查是否包含嵌套的<span>\n",
    "                    try:\n",
    "                        molecule_element = cells[1].find_element(By.CSS_SELECTOR, \"a, span\")  # 同时匹配链接或纯文本\n",
    "                        molecule_name = molecule_element.text.strip()\n",
    "                    except NoSuchElementException:\n",
    "                        molecule_name = \"N/A\"\n",
    "                    \n",
    "                    # ========== 修复点2: OB/DL列索引确认 ==========\n",
    "                    ob = cells[6].text.strip() if len(cells) > 6 else \"N/A\"\n",
    "                    dl = cells[9].text.strip() if len(cells) > 9 else \"N/A\"\n",
    "                    \n",
    "                    # ========== 修复点3: Targets提取逻辑 ==========\n",
    "                    targets = []\n",
    "                    try:\n",
    "                        target_link = cells[1].find_element(By.CSS_SELECTOR, \"a\").get_attribute(\"href\")\n",
    "                        driver.execute_script(f\"window.open('{target_link}');\")\n",
    "                        driver.switch_to.window(driver.window_handles[1])\n",
    "                        \n",
    "                        WebDriverWait(driver, 30).until(\n",
    "                            EC.presence_of_element_located((By.CSS_SELECTOR, \"td[role='gridcell'] > a[href^='target.php']\"))\n",
    "                        )\n",
    "                        \n",
    "                        # 精准定位靶基因名称（根据提供的HTML结构）\n",
    "                        target_elements = driver.find_elements(\n",
    "                            By.CSS_SELECTOR, \n",
    "                            \"td[role='gridcell'] > a[href^='target.php']\"  # 直接定位靶基因链接\n",
    "                        )\n",
    "                        targets = [el.text.strip() for el in target_elements if el.text.strip()]\n",
    "                        \n",
    "                        driver.close()\n",
    "                        driver.switch_to.window(driver.window_handles[0])\n",
    "                    except (NoSuchElementException, TimeoutException):\n",
    "                        if len(driver.window_handles) > 1:\n",
    "                            driver.close()\n",
    "                            driver.switch_to.window(driver.window_handles[0])\n",
    "                        targets = [\"N/A\"]\n",
    "                    \n",
    "                    all_data.append({\n",
    "                        \"Mol ID\": mol_id,\n",
    "                        \"Molecule Name\": molecule_name,\n",
    "                        \"OB (%)\": ob,\n",
    "                        \"DL\": dl,\n",
    "                        \"Targets\": \", \".join(targets) if targets else \"N/A\"\n",
    "                    })\n",
    "                    \n",
    "                except Exception as e:\n",
    "                    print(f\"提取失败: {str(e)}\")\n",
    "                    continue\n",
    "            \n",
    "            # 翻页逻辑保持不变\n",
    "            try:\n",
    "                next_btn = WebDriverWait(driver, 30).until(\n",
    "                    EC.element_to_be_clickable((By.CSS_SELECTOR, \"a.k-link[title='Go to the next page']:not(.k-state-disabled)\"))\n",
    "                )\n",
    "                next_btn.click()\n",
    "                WebDriverWait(driver, 30).until(EC.staleness_of(rows[0]))\n",
    "                current_page += 1\n",
    "            except Exception as e:\n",
    "                print(f\"翻页失败: {str(e)}\")\n",
    "                break\n",
    "        \n",
    "        return pd.DataFrame(all_data)\n",
    "    \n",
    "    finally:\n",
    "        driver.quit()\n",
    "\n",
    "# 执行抓取\n",
    "target_url = \"https://www.tcmsp-e.com/tcmspsearch.php?qr=Poria%20Cocos(Schw.)%20Wolf.&qsr=herb_en_name&token=940110e98cc8ed0273f506a81d47c346\"\n",
    "df = scrape_tcmsp(target_url, max_pages=64)\n",
    "\n",
    "# 保存数据\n",
    "df.to_csv(\"Hedysarum_Components_Corrected茯苓.csv\", index=False)\n",
    "print(\"数据已保存至 Hedysarum_Components_Corrected茯苓.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1c84bb0d-576e-4739-8305-5af9b5db3633",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
